{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\anaconda\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "F:\\anaconda\\lib\\site-packages\\ipykernel_launcher.py:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "F:\\anaconda\\lib\\site-packages\\pandas\\core\\indexing.py:543: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.model_selection import StratifiedKFold, train_test_split\n",
    "from matplotlib import pyplot as plt\n",
    "import config\n",
    "import pickle\n",
    "from DataLoader import FeatureDictionary, DataParser\n",
    "from DCN import DCN\n",
    "from metrics import Logloss\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "def load_data():\n",
    "    ## 读取特征矩阵\n",
    "    with open('../data/temp.pkl', 'rb') as file:\n",
    "        data = pickle.load(file)\n",
    "    \n",
    "    cols = [c for c in data.columns if c not in ['instance_id','click']]#提取特征集合\n",
    "    cols = [c for c in cols if (not c in config.IGNORE_COLS)]\n",
    "    \n",
    "    ### 为了测量结果，使用Logloss衡量，且将训练集划分为：训练集+测试集+验证集(按照时间划分)\n",
    "    ### period小于33的为训练集，period=33随即划分为验证集+测试集\n",
    "    total_train = data[data.click != -1]\n",
    "    \n",
    "    #将数值特征归一化\n",
    "    if config.NUMERIC_COLS != []:\n",
    "        mms = MinMaxScaler()\n",
    "        total_train[config.NUMERIC_COLS] = mms.fit_transform(total_train[config.NUMERIC_COLS])\n",
    "    \n",
    "    train = total_train[total_train.period <= 32][cols+['instance_id']]\n",
    "    train_y = total_train[total_train.period <= 32]['click'] ##标签\n",
    "    val_and_test = total_train[total_train.period == 33][cols+['instance_id']]\n",
    "    val_and_test_y = total_train[total_train.period == 33]['click']\n",
    "    val, test, val_y, test_y = train_test_split(val_and_test, val_and_test_y, test_size=0.5, random_state=1024)\n",
    "\n",
    "    dfTrain_fea = pd.concat((train, val), axis = 0)\n",
    "    dfTrain_y = pd.concat((train_y, val_y), axis = 0)\n",
    "    dfTrain = pd.concat((dfTrain_fea, dfTrain_y), axis = 1)\n",
    "    dfTest = pd.concat((test, test_y), axis = 1)\n",
    "\n",
    "\n",
    "\n",
    "    X_train = dfTrain[cols].values\n",
    "    y_train = dfTrain['click'].values\n",
    "\n",
    "    X_test = dfTest[cols].values\n",
    "    ids_test = dfTest['instance_id'].values\n",
    "\n",
    "    cat_features_indices = [i for i,c in enumerate(cols) if c in config.CATEGORICAL_COLS]\n",
    "\n",
    "    return dfTrain,dfTest,X_train,y_train,val, X_test,ids_test,cat_features_indices\n",
    "\n",
    "# load data\n",
    "dfTrain, dfTest, X_train, y_train, val, X_test, ids_test, cat_features_indices = load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcn_params = {\n",
    "\n",
    "    \"embedding_size\": 8,\n",
    "    \"deep_layers\": [32, 32],\n",
    "    \"dropout_deep\": [0.5, 0.5, 0.5],\n",
    "    \"deep_layers_activation\": tf.nn.relu,\n",
    "    \"epoch\": 30,\n",
    "    \"batch_size\": 256,\n",
    "    \"learning_rate\": 0.001,\n",
    "    \"optimizer_type\": \"adam\",\n",
    "    \"batch_norm\": 1,\n",
    "    \"batch_norm_decay\": 0.995,\n",
    "    \"l2_reg\": 0.01,\n",
    "    \"verbose\": True,\n",
    "    \"random_seed\": config.RANDOM_SEED,\n",
    "    \"cross_layer_num\":3,\n",
    "    \"eval_metric\":Logloss,\n",
    "    'greater_is_better':False,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_base_model_dcn(dfTrain,dfTest,dcn_params):\n",
    "    fd = FeatureDictionary(dfTrain,dfTest,\n",
    "                           numeric_cols=config.NUMERIC_COLS,\n",
    "                           ignore_cols = config.IGNORE_COLS)\n",
    "    data_parser = DataParser(feat_dict= fd)\n",
    "    # Xi_train ：列的序号\n",
    "    # Xv_train ：列的对应的值\n",
    "    data_parser = DataParser(feat_dict=fd)\n",
    "    Xi_train, Xv_train, numeric_Xv_train,y_train = data_parser.parse(df=dfTrain, has_label=True)\n",
    "    Xi_test, Xv_test, numeric_Xv_test,ids_test = data_parser.parse(df=dfTest)\n",
    "\n",
    "    dcn_params[\"cate_feature_size\"] = fd.feat_dim\n",
    "    dcn_params[\"field_size\"] = len(Xi_train[0])\n",
    "    dcn_params['numeric_feature_size'] = len(config.NUMERIC_COLS)\n",
    "\n",
    "    print(dfTrain.dtypes)\n",
    "    #将Xi_train分为训练集+验证集\n",
    "    Xi_train_, Xv_train_, numeric_Xv_train_, y_train_ =  Xi_train[:-val.shape[0]], Xv_train[:-val.shape[0]],numeric_Xv_train[:-val.shape[0]], y_train[:-val.shape[0]]\n",
    "    Xi_valid_, Xv_valid_, numeric_Xv_valid_, y_valid_ =  Xi_train[-val.shape[0]:], Xv_train[-val.shape[0]:], numeric_Xv_train[-val.shape[0]:], y_train[-val.shape[0]:]\n",
    "\n",
    "    \n",
    "    y_val_meta = np.zeros((val.shape[0],1),dtype=float)\n",
    "    y_test_meta = np.zeros((dfTest.shape[0],1),dtype=float)\n",
    "    #开始训练\n",
    "    dcn = DCN(**dcn_params)\n",
    "    dcn.fit(Xi_train_, Xv_train_, numeric_Xv_train_,y_train_, Xi_valid_, Xv_valid_, numeric_Xv_valid_,y_valid_, early_stopping=True)\n",
    "    y_val_meta[:,0] += dcn.predict(Xi_valid_, Xv_valid_, numeric_Xv_valid_)  #预测验证集\n",
    "    losses = Logloss(y_valid_, y_val_meta[:,0])##验证集loss\n",
    "    print('验证集loss为: %.4f' %losses)\n",
    "    \n",
    "    y_test_meta[:,0] += dcn.predict(Xi_test, Xv_test, numeric_Xv_test)  #预测测试集\n",
    "\n",
    "    filename = \"%s_loss%.4f.csv\"%('DCN', losses)\n",
    "    _make_submission(ids_test, y_test_meta, filename)\n",
    "#     _plot_fig(gini_results_epoch_train, gini_results_epoch_valid, clf_str)\n",
    "\n",
    "    return y_test_meta\n",
    "\n",
    "def _make_submission(ids, y_pred, filename=\"submission.csv\"):\n",
    "    pd.DataFrame({\"instance_id\": ids, \"click\": y_pred.flatten()}).to_csv(\n",
    "        os.path.join(config.SUB_DIR, filename), index=False, float_format=\"%.5f\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始训练\n",
      "adid                            int64\n",
      "advert_industry_inner           int64\n",
      "advert_name                     int64\n",
      "app_cate_id                     int64\n",
      "app_id                          int64\n",
      "campaign_id                     int64\n",
      "carrier                         int64\n",
      "city                            int64\n",
      "creative_has_deeplink           int64\n",
      "creative_height                 int64\n",
      "creative_id                     int64\n",
      "creative_is_download            int64\n",
      "creative_is_jump                int64\n",
      "creative_tp_dnf                 int64\n",
      "creative_type                   int64\n",
      "creative_width                  int64\n",
      "devtype                         int64\n",
      "f_channel                       int64\n",
      "inner_slot_id                   int64\n",
      "nnt                             int64\n",
      "orderid                         int64\n",
      "os                              int64\n",
      "province                        int64\n",
      "sim_ip                          int64\n",
      "clear_make                      int64\n",
      "clear_model                     int64\n",
      "clear_osv                       int64\n",
      "hour                          float64\n",
      "advert_id_rate                float64\n",
      "advert_industry_inner_rate    float64\n",
      "                               ...   \n",
      "creative_tp_dnf_rate          float64\n",
      "creative_width_rate           float64\n",
      "province_rate                 float64\n",
      "f_channel_rate                float64\n",
      "adid_model_nuq_num            float64\n",
      "model_adid_nuq_num            float64\n",
      "adid_make_nuq_num             float64\n",
      "make_adid_nuq_num             float64\n",
      "adid_os_nuq_num               float64\n",
      "os_adid_nuq_num               float64\n",
      "adid_city_nuq_num             float64\n",
      "city_adid_nuq_num             float64\n",
      "adid_province_nuq_num         float64\n",
      "province_adid_nuq_num         float64\n",
      "adid_f_channel_nuq_num        float64\n",
      "f_channel_adid_nuq_num        float64\n",
      "adid_app_id_nuq_num           float64\n",
      "app_id_adid_nuq_num           float64\n",
      "adid_carrier_nuq_num          float64\n",
      "carrier_adid_nuq_num          float64\n",
      "adid_nnt_nuq_num              float64\n",
      "nnt_adid_nuq_num              float64\n",
      "adid_devtype_nuq_num          float64\n",
      "devtype_adid_nuq_num          float64\n",
      "adid_app_cate_id_nuq_num      float64\n",
      "app_cate_id_adid_nuq_num      float64\n",
      "adid_inner_slot_id_nuq_num    float64\n",
      "inner_slot_id_adid_nuq_num    float64\n",
      "instance_id                     int64\n",
      "click                         float64\n",
      "Length: 63, dtype: object\n",
      "#params: 71630\n",
      "[1] train-result=0.4167, valid-result=0.4308 [713.9 s]\n",
      "[2] train-result=0.4162, valid-result=0.4307 [745.8 s]\n",
      "[3] train-result=0.4174, valid-result=0.4332 [734.9 s]\n",
      "[4] train-result=0.4169, valid-result=0.4323 [724.3 s]\n",
      "[5] train-result=0.4154, valid-result=0.4316 [713.9 s]\n",
      "[6] train-result=0.4159, valid-result=0.4318 [743.0 s]\n",
      "验证集loss为: 0.4318\n"
     ]
    }
   ],
   "source": [
    "# # ------------------ DCN Model ------------------\n",
    "##y_train_dfm,y_test_dfm = run_base_model_dfm(dfTrain,dfTest,folds,dfm_params)\n",
    "print('开始训练')\n",
    "y_test_dfm = run_base_model_dcn(dfTrain, dfTest, dcn_params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 测试集loss为： 0.4284\n"
     ]
    }
   ],
   "source": [
    "### 评判测试集的logloss\n",
    "print(' 测试集loss为： %.4f' %Logloss(dfTest['click'].values, y_test_dfm[:,0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
